{"componentChunkName":"component---src-templates-bootcamp-js","path":"/bootcamp/bigdata/201-kafka-4/","result":{"data":{"site":{"siteMetadata":{"title":"William Blogs & More","description":"team knowledge base"}},"allBootcamp":{"edges":[{"node":{"fields":{"id":"410a853e-19db-5ecf-9193-d90fd4e2b164","slug":"/README/","title":"README"}}},{"node":{"fields":{"id":"ebd8a7a9-4f93-5f7f-89b4-7a921fc850fd","slug":"/backend/DEV/","title":"DEV"}}},{"node":{"fields":{"id":"c97bf13a-dd3f-5e0b-9f0a-9c5f08a42d57","slug":"/backend/Restful/","title":"Restful"}}},{"node":{"fields":{"id":"81e9e69e-bf6a-51e5-843d-7b4321c330f9","slug":"/backend/curl/","title":"Curl"}}},{"node":{"fields":{"id":"2aa2ab91-e4c3-59d3-ae10-2949c78bbba2","slug":"/backend/express/","title":"Express"}}},{"node":{"fields":{"id":"923aee4e-1ea6-5b5c-9dbf-7f3ed2ccd8de","slug":"/backend/flow/","title":"Flow"}}},{"node":{"fields":{"id":"735e010f-fe92-58c2-a717-44e1379ec81a","slug":"/backend/microservice/","title":"Microservice"}}},{"node":{"fields":{"id":"5257ef00-003a-505c-9681-90b156b2fdc3","slug":"/backend/mongo/","title":"Mongo"}}},{"node":{"fields":{"id":"5d7e4d14-bbe0-51e7-ac00-aed822be865d","slug":"/backend/nodejs/","title":"Nodejs"}}},{"node":{"fields":{"id":"04e0f6cd-ba6d-5bcb-a711-f978053c85a9","slug":"/backend/pg/","title":"Pg"}}},{"node":{"fields":{"id":"07933914-1441-5d9a-93b3-7b8527f32834","slug":"/backend/rabbitmq/","title":"Rabbitmq"}}},{"node":{"fields":{"id":"aeee00c3-e65a-5e57-8e26-99bdee4b5972","slug":"/backend/redis/","title":"Redis"}}},{"node":{"fields":{"id":"5efa4932-4a43-5627-a52e-d6c44ea1ce37","slug":"/backend/socket/","title":"Socket"}}},{"node":{"fields":{"id":"2546451c-541b-5d1f-b34c-1f1842685cfc","slug":"/bestitconsulting/Bootcamp/1-bootcamp/","title":"1 Bootcamp"}}},{"node":{"fields":{"id":"3ce1ed0e-ce4c-53b6-89a3-dd5dc7642647","slug":"/bestitconsulting/Bootcamp/2-release_branch/","title":"2 Release_branch"}}},{"node":{"fields":{"id":"e3f26574-c4e6-5058-8336-6e761d369455","slug":"/bestitconsulting/Bootcamp/3-coding/","title":"3 Coding"}}},{"node":{"fields":{"id":"8b752412-b3d3-5bbe-87b3-d99ef91d2759","slug":"/bestitconsulting/Bootcamp/4-testing/","title":"4 Testing"}}},{"node":{"fields":{"id":"e950ea75-d6c1-5fca-8e1c-4b4fce362fab","slug":"/bestitconsulting/Bootcamp/5-CICD/","title":"5 CICD"}}},{"node":{"fields":{"id":"739b0238-56b4-5a47-bc76-3286eadcb425","slug":"/bestitconsulting/Bootcamp/6-BA/","title":"6 BA"}}},{"node":{"fields":{"id":"bc2e6ad3-6b39-54b0-8d66-e5e295f331ee","slug":"/bestitconsulting/Bootcamp/QA/","title":"QA"}}},{"node":{"fields":{"id":"484882b7-848a-591f-80d5-24d045cb9582","slug":"/bestitconsulting/Bootcamp/README/","title":"README"}}},{"node":{"fields":{"id":"27252ae9-1261-5b83-9039-45c41697c09d","slug":"/bestitconsulting/Bootcamp/Reference/","title":"Reference"}}},{"node":{"fields":{"id":"2a271bc5-0775-5cb0-99ab-056d5c0cf09f","slug":"/bestitconsulting/courses/agile/","title":"Agile"}}},{"node":{"fields":{"id":"372866f9-ffd2-5576-bbc1-415732492686","slug":"/bestitconsulting/courses/grow-admin/","title":"Grow Admin"}}},{"node":{"fields":{"id":"5eb7bdc4-c171-52bc-bf48-f413de03ea24","slug":"/bestitconsulting/courses/learn-suite/","title":"Learn Suite"}}},{"node":{"fields":{"id":"986b6381-1b35-5253-93d5-1294d4984202","slug":"/bestitconsulting/courses/security/","title":"Security"}}},{"node":{"fields":{"id":"8e3b3436-48d9-57f8-9d53-71494461a76e","slug":"/bestitconsulting/radar/github-radar-1/","title":"Github Radar 1"}}},{"node":{"fields":{"id":"f9b1b128-a118-5569-b2ee-b6d5c347e964","slug":"/bestitconsulting/radar/github-radar/","title":"Github Radar"}}},{"node":{"fields":{"id":"17a02ec4-b391-54f1-8c60-922492f1a5eb","slug":"/bigdata/1-bigdata/","title":"1 Bigdata"}}},{"node":{"fields":{"id":"36884170-cd04-548a-9a00-77e36a4b1380","slug":"/bigdata/2-hadoop/","title":"2 Hadoop"}}},{"node":{"fields":{"id":"c9cdff49-84d1-58e6-bf25-926b7f68d84c","slug":"/bigdata/201-kafka-4/","title":"201 Kafka 4"}}},{"node":{"fields":{"id":"f2387a56-a470-54c8-94aa-182fffbab8b0","slug":"/bigdata/201-nosql-8/","title":"201 Nosql 8"}}},{"node":{"fields":{"id":"4d3d0dd0-c311-51f6-9975-80fd4c5fa752","slug":"/bigdata/201-streaming-5/","title":"201 Streaming 5"}}},{"node":{"fields":{"id":"27a22e9e-6aa2-5c4e-9edf-c06b8740246d","slug":"/bigdata/3-hdfs/","title":"3 Hdfs"}}},{"node":{"fields":{"id":"f80ad3db-5700-5e3b-a5ae-2e6f5dc0c886","slug":"/bigdata/4-devops/","title":"4 Devops"}}},{"node":{"fields":{"id":"07672f34-d70e-589c-ba8e-67e284bc1d8c","slug":"/bigdata/5-hive/","title":"5 Hive"}}},{"node":{"fields":{"id":"bf3a9766-31fd-5213-8dd9-7c38ebe7bb80","slug":"/bigdata/6-spark/","title":"6 Spark"}}},{"node":{"fields":{"id":"4dab467f-b6ea-531b-8c28-9bcef71863e4","slug":"/bigdata/7-docker/","title":"7 Docker"}}},{"node":{"fields":{"id":"7ee4ff75-d3f8-5c1a-b701-facf126f3450","slug":"/bigdata/8-streaming/","title":"8 Streaming"}}},{"node":{"fields":{"id":"6785ce66-99e5-5e00-af30-b1c32c61cb12","slug":"/bigdata/9-elasticsearch/","title":"9 Elasticsearch"}}},{"node":{"fields":{"id":"9a59f47c-4cd3-51db-9962-2ffb26b14dc4","slug":"/bigdata/hdfs-kubernetes/","title":"Hdfs Kubernetes"}}},{"node":{"fields":{"id":"819a354c-d042-54db-9466-5063d21db45f","slug":"/bigdata/kubernetes/","title":"Kubernetes"}}},{"node":{"fields":{"id":"4535d21c-0f79-5cda-b235-98658a1ffe1f","slug":"/bigdata/spark-kubernetes/","title":"Spark Kubernetes"}}},{"node":{"fields":{"id":"07d1d0ca-dc16-52b5-b911-0d128a2388eb","slug":"/bigdata2/README/","title":"README"}}},{"node":{"fields":{"id":"605faba4-a685-5a80-a80a-5130700509e3","slug":"/bigdata2/TODO/","title":"TODO"}}},{"node":{"fields":{"id":"5b1a9498-e94f-5dba-a4fa-9aa2f3da4312","slug":"/bigdata2/editor/","title":"Editor"}}},{"node":{"fields":{"id":"6e1973a3-4045-57b9-a096-d498f5d2802b","slug":"/bigdata2/hadoop/","title":"Hadoop"}}},{"node":{"fields":{"id":"e63f21ad-65e0-56db-891e-60a89c5f93da","slug":"/bigdata2/kafka/","title":"Kafka"}}},{"node":{"fields":{"id":"2efc1fee-93bd-53eb-b5e0-672ff2acc9a4","slug":"/bigdata2/scala/","title":"Scala"}}},{"node":{"fields":{"id":"001e1d27-4cba-5926-ba33-8197e2065136","slug":"/bigdata2/spark/","title":"Spark"}}},{"node":{"fields":{"id":"2e2078b0-b734-5ae3-873c-022349c1c3cf","slug":"/bigdata2/vscode/","title":"Vscode"}}},{"node":{"fields":{"id":"b52de8e7-4791-556a-9989-b84717cd3370","slug":"/bigdata2/web-resource/","title":"Web Resource"}}},{"node":{"fields":{"id":"e0378d10-6064-560e-a6b2-cbe89a820d26","slug":"/bigdata2/zookeeper/","title":"Zookeeper"}}},{"node":{"fields":{"id":"616110f5-9073-5438-846b-ff7584d4e922","slug":"/cloud/aws/","title":"Aws"}}},{"node":{"fields":{"id":"c788368a-0f0e-5661-8fe9-c469d5a278d8","slug":"/cloud/azure/","title":"Azure"}}},{"node":{"fields":{"id":"30bec85f-fc28-5638-b42d-a952b414de7c","slug":"/cloud/elk/","title":"Elk"}}},{"node":{"fields":{"id":"57773e69-6a69-5e86-a641-af37ffe1ad40","slug":"/cloud/gcp/","title":"Gcp"}}},{"node":{"fields":{"id":"6688fe10-a20c-5479-bc80-170063d3e58c","slug":"/frontend/apollo/","title":"Apollo"}}},{"node":{"fields":{"id":"09e2c05f-9d10-5776-8fe8-44e9a565a20e","slug":"/frontend/graphql/","title":"Graphql"}}},{"node":{"fields":{"id":"28d38d20-c6e2-50ae-ac83-d3e3b9f25248","slug":"/frontend/proxy/","title":"Proxy"}}},{"node":{"fields":{"id":"ca4444ea-9796-5d40-98e1-c01d80f88221","slug":"/frontend/react-router/","title":"React Router"}}},{"node":{"fields":{"id":"61e06420-e9e4-5aa4-9d71-bf618be3f4fe","slug":"/frontend/react/","title":"React"}}},{"node":{"fields":{"id":"9d95393b-96be-5065-83bb-536b746eed21","slug":"/frontend/redux/","title":"Redux"}}},{"node":{"fields":{"id":"e3bb0f44-4819-556c-8511-7402303e36c5","slug":"/misc/0428/","title":"0428"}}},{"node":{"fields":{"id":"dfcc8a86-17af-5c8c-b831-9fe4fa51bf0e","slug":"/misc/HISTORY/","title":"HISTORY"}}},{"node":{"fields":{"id":"6f004260-a55d-5a2d-8146-d197509fd980","slug":"/misc/TODO/","title":"TODO"}}},{"node":{"fields":{"id":"30096e9c-2593-5fc8-80d5-1f7648898888","slug":"/misc/git/","title":"Git"}}},{"node":{"fields":{"id":"bed21539-b0d7-5936-9c15-96905db68ac9","slug":"/misc/gitlab/","title":"Gitlab"}}},{"node":{"fields":{"id":"ab43544c-0ef1-56d2-a0a3-035fdd1cf7fe","slug":"/misc/installation/","title":"Installation"}}},{"node":{"fields":{"id":"b2e02e77-9623-5967-a541-fe1829315b47","slug":"/misc/misc/","title":"Misc"}}},{"node":{"fields":{"id":"4c7d96dc-fddc-52fd-9333-9fa6d2633874","slug":"/misc/vocabulary/","title":"Vocabulary"}}},{"node":{"fields":{"id":"f2a8cc1a-8b27-541c-b68e-8c5dc6d0f6b3","slug":"/poc/courses/agile/","title":"Agile"}}},{"node":{"fields":{"id":"ae6e68d8-0f75-5fac-ad96-f042bddd50aa","slug":"/poc/courses/grow-admin/","title":"Grow Admin"}}},{"node":{"fields":{"id":"689b1b72-e7b6-5b07-9890-50f653fdd7df","slug":"/poc/courses/learn-suite/","title":"Learn Suite"}}},{"node":{"fields":{"id":"003c5b81-20aa-57aa-b268-17aaaa338e68","slug":"/poc/courses/security/","title":"Security"}}},{"node":{"fields":{"id":"e0f53ac9-fbe0-5c72-8b69-ee2cce387f90","slug":"/poc/radar/github-radar-1/","title":"Github Radar 1"}}},{"node":{"fields":{"id":"c5718e38-5cc1-566e-a649-a4d501877995","slug":"/poc/radar/github-radar/","title":"Github Radar"}}},{"node":{"fields":{"id":"5a68cfb3-1b53-5527-a6ad-b8277381fab4","slug":"/poc/radar/steps/","title":"Steps"}}}]},"bootcamp":{"fields":{"id":"c9cdff49-84d1-58e6-bf25-926b7f68d84c","title":"201 Kafka 4","slug":"/bigdata/201-kafka-4/","tag":"bigdata","category":"bootcamp","date":"2020-12-17","size":4836},"html":"<h1 id=\"kafka\" style=\"position:relative;\"><a href=\"#kafka\" aria-label=\"kafka permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Kafka</h1>\n<p>Apache Kafka is a distributed <code class=\"language-text\">streaming</code> <code class=\"language-text\">platform</code></p>\n<p><span\n      class=\"gatsby-resp-image-wrapper\"\n      style=\"position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 600px; \"\n    >\n      <a\n    class=\"gatsby-resp-image-link\"\n    href=\"/static/7f17c06192ec7829310baea95b17d18e/fe9f1/pub-sub-pattern.png\"\n    style=\"display: block\"\n    target=\"_blank\"\n    rel=\"noopener\"\n  >\n    <span\n    class=\"gatsby-resp-image-background-image\"\n    style=\"padding-bottom: 48%; position: relative; bottom: 0; left: 0; background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAKCAYAAAC0VX7mAAAACXBIWXMAAA7DAAAOwwHHb6hkAAABpklEQVQoz31Sy0rDUBTMj7hzIYgPxI/UjboSca3gW4ugWGqLom6s+EKjVdOaxObVpOkrSZsm48ltWoJVDwyXO5w7OZk5HKjCMBxCnw9idGM+CIIBfnvHJR8PxBKiyfK7AQzDgK7rUBQFjusyPkj0cP0Jks+DWOyr2cK+IGPzXcKzaTOuXm+QaIXOOjzfx6tl41pWcKMaqHhtcH2RaqUCWZJgWdZA/PCzjOm9HCZ2s5i/e+31MUEDNRKsuR7yio40X0C2KEKhOyc1WpAjUGPTcSGRaOlTBLo+DktlzKTOMX5whqUnAY5l4onnoWnakB1hbBuXei4g9cjjsqzBIY9u83nkMhm4zQY2XgRMraxjbHkNi7cvKAsfODk6gizLA9+RRORhtd2BTah3fEY4jgOP7lEdiypmdzKY3Epj4eENaHvMw7+KTfiTiKobn9Evj26fYmQrg7nYwwZ90DRNFopLQzzoJi5KEq4oGM1t99bmJ/opa+Tp+puIVb4I3qqhQ6maVZuFEonaLQf3JJgtCLgQv3qC/40/tIe0zNEOqqrK9tCN9zBMRPMNhULx0WdOq9sAAAAASUVORK5CYII='); background-size: cover; display: block;\"\n  ></span>\n  <img\n        class=\"gatsby-resp-image-image\"\n        alt=\"Publish/Subscribe pattern\"\n        title=\"Publish/Subscribe pattern\"\n        src=\"/static/7f17c06192ec7829310baea95b17d18e/0a47e/pub-sub-pattern.png\"\n        srcset=\"/static/7f17c06192ec7829310baea95b17d18e/8a4e8/pub-sub-pattern.png 150w,\n/static/7f17c06192ec7829310baea95b17d18e/5a46d/pub-sub-pattern.png 300w,\n/static/7f17c06192ec7829310baea95b17d18e/0a47e/pub-sub-pattern.png 600w,\n/static/7f17c06192ec7829310baea95b17d18e/fe9f1/pub-sub-pattern.png 883w\"\n        sizes=\"(max-width: 600px) 100vw, 600px\"\n        style=\"width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;\"\n        loading=\"lazy\"\n      />\n  </a>\n    </span></p>\n<h2 id=\"what-is-kafka\" style=\"position:relative;\"><a href=\"#what-is-kafka\" aria-label=\"what is kafka permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>What is Kafka</h2>\n<ul>\n<li>Apache Kafka is a fast, scalable, durable, and fault-tolerant publish-subscribe messaging system</li>\n<li><b>Kafka is often used in place of traditional message brokers like <code class=\"language-text\">JMS</code> and <code class=\"language-text\">AMQP</code></b> because of its higher throughput, reliability and replication</li>\n<li>Kafka may work in combination with Storm, Spark, Samza, Flink, etc. for real-time analysis and rendering of streaming data</li>\n<li>Whatever the industry or use case, Kafka brokers massive message streams for low-latency analysis in Enterprise Apache Hadoop</li>\n</ul>\n<h2 id=\"what-does-kafka-do\" style=\"position:relative;\"><a href=\"#what-does-kafka-do\" aria-label=\"what does kafka do permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>What does Kafka do?</h2>\n<p>Apache Kafka supports a wide range of use cases where high throughput, reliable delivery, and horizontal scalability are important. <code class=\"language-text\">Apache Spark</code> and <code class=\"language-text\">Apache Cassandra</code> work very well in combination with <code class=\"language-text\">Kafka</code>.</p>\n<p>Typical use cases include:</p>\n<ul>\n<li>Messaging</li>\n<li>Stream processing: pipeline, transfer, aggregate, lightweight library <code class=\"language-text\">kafka stream</code>, in place of Apache storm, Apache Samsome</li>\n<li>Metrics collection &#x26; monitoring</li>\n<li>Website activity tracking: pageviews, searching, user activity capture to topic</li>\n<li>Event sourcing (CDC)</li>\n<li>Log aggregation</li>\n<li>Commit log (log replication)</li>\n</ul>\n<p><span\n      class=\"gatsby-resp-image-wrapper\"\n      style=\"position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 355px; \"\n    >\n      <a\n    class=\"gatsby-resp-image-link\"\n    href=\"/static/bbae500be094fa4a268c3b7422db28ad/526ee/kafka2.png\"\n    style=\"display: block\"\n    target=\"_blank\"\n    rel=\"noopener\"\n  >\n    <span\n    class=\"gatsby-resp-image-background-image\"\n    style=\"padding-bottom: 103.33333333333331%; position: relative; bottom: 0; left: 0; background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAVCAYAAABG1c6oAAAACXBIWXMAAA7DAAAOwwHHb6hkAAACY0lEQVQ4y2WUW2/bMAyF/f//1N76UGxtgWHAMKDrdV3iOq4vie8yx08WXdkj4MhSSOrw8JiJqM3zvHmcPnU/yDA5qXQdnZNmnKTVx9Z9nO0T29gBNrlZPrpenquzT3geRjleWnmpLvJaX+TUdrI3i034ARFB3GwrCLtp2fe6XsLKeT0Mft+M4+q/IuSl19Lezo08lmePAmcMlOwxgv+qD+UTmCriP4oW5IX6bRLCFWU8aUIcKNOFEtKm9Wve9msgALKm8wkP6g/KTclsKI/EfUiOM2hAQjDo4ZYzUOPDu8VtEbolkYsaxPtDWUvZ94GrwSMkWdzI0cdOa2ziibZGaJAl5pJSS6fk2/TkkxeanKaQcJHSZ1PqUHZCA3CCcJJbOcgjV0Rfj5l8eT7I1Vvqz7gAGmhirn5cCu+UTrxHuEjDbchlzyWZBt0XlZLf+K5zZtbt5LRyCHxgG7mUctBAOIMGeAQ1e1YQQYldTOJNl0mII3JBHr+L2p9hJiMuBCVWBKSgKrtF6P8Jm4Qkem8+b2dFPmbGlb1/O548HWUodyMbbiIpCBkAc/gc4++WQNCiR5qHNum++awJTQbx894snPFQAWeUZt87e+RFcrefNrFQbdrwGd5oSdBgQ+JXXsr1IfP/TbuYTVP2xo2UBulIhQtAie4QMOhdNDvjuZgYbCs1HpZ9aEBsJN5XFFtCE+7SXH5khXzPPuTnqVz1aCMMpGiTpthow+DTZFUFCSU4PulkZjrbWocgkCAbzujoYxhvZvB6rxzbf3wMCWXyB9+nHU7Obcrg5hdNjJibaDobBWjX9PoPZlho6k5+bSAAAAAASUVORK5CYII='); background-size: cover; display: block;\"\n  ></span>\n  <img\n        class=\"gatsby-resp-image-image\"\n        alt=\"Kafaka\"\n        title=\"Kafaka\"\n        src=\"/static/bbae500be094fa4a268c3b7422db28ad/526ee/kafka2.png\"\n        srcset=\"/static/bbae500be094fa4a268c3b7422db28ad/8a4e8/kafka2.png 150w,\n/static/bbae500be094fa4a268c3b7422db28ad/5a46d/kafka2.png 300w,\n/static/bbae500be094fa4a268c3b7422db28ad/526ee/kafka2.png 355w\"\n        sizes=\"(max-width: 355px) 100vw, 355px\"\n        style=\"width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;\"\n        loading=\"lazy\"\n      />\n  </a>\n    </span></p>\n<h3 id=\"qualities\" style=\"position:relative;\"><a href=\"#qualities\" aria-label=\"qualities permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Qualities</h3>\n<ul>\n<li>Scalability</li>\n<li>Durability (耐久力)</li>\n<li>Reliability</li>\n<li>Performance</li>\n</ul>\n<h3 id=\"high-level-overview\" style=\"position:relative;\"><a href=\"#high-level-overview\" aria-label=\"high level overview permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>High-level overview</h3>\n<p><span\n      class=\"gatsby-resp-image-wrapper\"\n      style=\"position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 554px; \"\n    >\n      <a\n    class=\"gatsby-resp-image-link\"\n    href=\"/static/401fb95d56481091373e7751285eebb1/04abd/kafka3.png\"\n    style=\"display: block\"\n    target=\"_blank\"\n    rel=\"noopener\"\n  >\n    <span\n    class=\"gatsby-resp-image-background-image\"\n    style=\"padding-bottom: 68.66666666666667%; position: relative; bottom: 0; left: 0; background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAOCAYAAAAvxDzwAAAACXBIWXMAAA7DAAAOwwHHb6hkAAACb0lEQVQ4y21S227TQBD1j/Mn/AM0rfKABEhUQjxQoE0dp6lvcRLbsRPf1pf4lsPZxIQUutJodudyZs7MKtuygltU8Mv9UTyKm5co6gYetTfYpci4kCLqGmv6/AuffKf7GsrPMMKVZuBa0zGaznHzZOLj0sdK5Pi89HA1fcZIo6j0Pdv4HkSwU4Fr3kf0yTwpY3OF5ziDch/GGOsLjGc63t1PqQ3cuiG7LPBl5WM8N3HDQqPJDGOC/GC8k+X4YK8xfjLwnvYbFpNvIxFQZJvbao+IFFeigB4lcJIUOWlFtCdNiw1pbjiamDEyPqXditNjrFeUtLcI6c/pV/DvORxQN0wsSwgGHfoeEelL6bsO2WBv2xavHeVAACk9RYLtmSTaDgWBZHc7dnM9meLT3MBOdkdbLoGpuyH3UpRTU6eHPIKVfVJ00wzLKIaXCWw4T5djkDpkpyoX+VY1SLc65vQX+cpfpidDzu58JqqmhbuphgddxyIIoRomVMuGZpr4ajp4822CWZSeAf+jfO6QVLaSGoHTrseOg3b4TVYsIu+CNpf/7o7fp2DMJcMz5WEX59NwfkJuX4JQpA5I/c+75NJe7vGVDiViwi8Q5TliJgkuoxk26cUJQgLKsyeDVH6hguCcZ1ZVA2h/xFCaeIc63KCLtrAMHYm3RrsLcSgLNNsAHe/SlnkuY0K0cYRDIdAEPoLVEg5niuSE0WYplD0va02FNbmHN5vCeviFxLHRiwwZtf34gKU6oTzCZoyMb5MYW32OBX3ukOM/aejSGErNSuXSRs5k4VhHqdYOqyVHXSxPdunPFxbqwCNg9CJH6oK6Jdvfdzgjg6jHo3wAAAAASUVORK5CYII='); background-size: cover; display: block;\"\n  ></span>\n  <img\n        class=\"gatsby-resp-image-image\"\n        alt=\"Kafaka\"\n        title=\"Kafaka\"\n        src=\"/static/401fb95d56481091373e7751285eebb1/04abd/kafka3.png\"\n        srcset=\"/static/401fb95d56481091373e7751285eebb1/8a4e8/kafka3.png 150w,\n/static/401fb95d56481091373e7751285eebb1/5a46d/kafka3.png 300w,\n/static/401fb95d56481091373e7751285eebb1/04abd/kafka3.png 554w\"\n        sizes=\"(max-width: 554px) 100vw, 554px\"\n        style=\"width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;\"\n        loading=\"lazy\"\n      />\n  </a>\n    </span></p>\n<h3 id=\"kafka-architecture\" style=\"position:relative;\"><a href=\"#kafka-architecture\" aria-label=\"kafka architecture permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Kafka architecture</h3>\n<p><span\n      class=\"gatsby-resp-image-wrapper\"\n      style=\"position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 600px; \"\n    >\n      <a\n    class=\"gatsby-resp-image-link\"\n    href=\"/static/90ee323681dbdf702202d11583377d6b/54bf4/kafka4.png\"\n    style=\"display: block\"\n    target=\"_blank\"\n    rel=\"noopener\"\n  >\n    <span\n    class=\"gatsby-resp-image-background-image\"\n    style=\"padding-bottom: 41.333333333333336%; position: relative; bottom: 0; left: 0; background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAICAYAAAD5nd/tAAAACXBIWXMAAA7DAAAOwwHHb6hkAAABi0lEQVQoz1WRS2/UQBCE9/9fOILglkNElFsQPwCBcuJAJBQtbJyNPbHX8XP8GL/HH20rRpuRWjVd3a7pau+QUw4jN47i+v7Al+Mz9TiynWNZc7V3uLp/4HsQYa2lqmvCMOQUBAQSeZ5TC5emKbvlo0GaDrrkx5Pi1gvQTYvpetp+wC8qvh0VP30RMC12mqiMwQ9OeErx7PsowSVWwXmGXgQzaeplsqisOCYZXiqR5bhpTiEPdMOwPjTamaqqyOMIncTkSYLWeuWKomCnxe4+SvgdRkTdgJNp7mSaXHgldu/8E6oyBKbhlwpw65bCV7gX79l/fEd485nzs9P9iCNTeGI5k/uCj5InYtmvDY5MG9QNqeROnKBMhxZB7/ITzsUHTl+vmWeLGBWcF8szVhIzTrLLmUZwei0uTf0r102WRvZnhZ8WnOTHyarmJZa9bYLrTYpN8oJz+IuJX9Z8E5zbhli5PDkHplK/FTizunGroB16SveR9GG/ou37/w1jWZA7f9ZaE/pvpjmPjfsHkx1gzWmmuLcAAAAASUVORK5CYII='); background-size: cover; display: block;\"\n  ></span>\n  <img\n        class=\"gatsby-resp-image-image\"\n        alt=\"Kafaka\"\n        title=\"Kafaka\"\n        src=\"/static/90ee323681dbdf702202d11583377d6b/0a47e/kafka4.png\"\n        srcset=\"/static/90ee323681dbdf702202d11583377d6b/8a4e8/kafka4.png 150w,\n/static/90ee323681dbdf702202d11583377d6b/5a46d/kafka4.png 300w,\n/static/90ee323681dbdf702202d11583377d6b/0a47e/kafka4.png 600w,\n/static/90ee323681dbdf702202d11583377d6b/1cfc2/kafka4.png 900w,\n/static/90ee323681dbdf702202d11583377d6b/54bf4/kafka4.png 1007w\"\n        sizes=\"(max-width: 600px) 100vw, 600px\"\n        style=\"width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;\"\n        loading=\"lazy\"\n      />\n  </a>\n    </span></p>\n<h3 id=\"core-apis\" style=\"position:relative;\"><a href=\"#core-apis\" aria-label=\"core apis permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Core APIs</h3>\n<ol>\n<li>The <code class=\"language-text\">Producer API</code> allows an application to publish a stream of records to one or more Kafka topics. </li>\n<li>The <code class=\"language-text\">Consumer API</code> allows an application to subscribe to one or more topics and process the stream of records produced to them. </li>\n<li>The <code class=\"language-text\">Connect API</code> allows building and running reusable producers or consumers that connect Kafka topics to existing applications or data systems. For example, a connector to a relational database might capture every change to a table. </li>\n<li>The <code class=\"language-text\">Streams API</code> allows an application to act as a stream processor, consuming an input stream from one or more topics and producing an output stream to one or more output topics, transforming the input streams to output streams.</li>\n</ol>\n<p><span\n      class=\"gatsby-resp-image-wrapper\"\n      style=\"position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 493px; \"\n    >\n      <a\n    class=\"gatsby-resp-image-link\"\n    href=\"/static/38cc5152b7b77b65a9415670e114f610/f88d0/kafka5.png\"\n    style=\"display: block\"\n    target=\"_blank\"\n    rel=\"noopener\"\n  >\n    <span\n    class=\"gatsby-resp-image-background-image\"\n    style=\"padding-bottom: 83.33333333333334%; position: relative; bottom: 0; left: 0; background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAARCAYAAADdRIy+AAAACXBIWXMAAA7DAAAOwwHHb6hkAAAB+0lEQVQ4y4WUh6rrQAxE/f9fFRIS0nvvvffe9TgCGV8n4S0sa3u10sxovI74xvv91vV0Okmj0ZDBYCDD4VDG47H0ej05n88fsd7h+JNZ0GazkUwmI+l0WiKRiCQSCYlGozKbzXT/9Xp9Tep8QzcajXRut1u53+8ynU5lvV7L5XKRxWKhe8vl8gMEq+N9sar9fl9SqZRSPBwOirJSqchqtZJ2uy3FYtFF6gfjfIN9vV5Vq8lkokVAhwSPx0O1ZB/kxOz3e9X7AyFIQLDb7dzEx+NRD3GAyTvJbMCg0+loETehbUIlFospHZLfbjcNqNVqOsvlsnYdHUnMt2q1qg4wPWHjPJ9Pt6tUQhuSUYAu8y2ZTKqm8/lcn7ESiEOhkOTzedUYpKphNpuVcDisgaxMswf0WbEM6K3r5gQaVSqVJJfLKQD2HDSCgiEqFApKg0QcAiEo4vG42gfqgKAoyAKBgPo0GAzqeVdDaHCQaq1WS5CCQiSn20ySW+PQk3iK8B39VENvlzEtqw0o8E4jzCIWj4XQjcbA5E+X/T4kGHugnTmg2+1qHNRBDxoY4FGz2s8/BZr1el0TgRDdeOZws9lUZHT855/i/4hh0YdDWIhukgSroBed9V4Sf/7lX7cNhzEvaO2ygDbNMcr/vW28AXYp0HVsAWJWbAXqX/fhP+eqB0Nf5r9SAAAAAElFTkSuQmCC'); background-size: cover; display: block;\"\n  ></span>\n  <img\n        class=\"gatsby-resp-image-image\"\n        alt=\"Kafaka\"\n        title=\"Kafaka\"\n        src=\"/static/38cc5152b7b77b65a9415670e114f610/f88d0/kafka5.png\"\n        srcset=\"/static/38cc5152b7b77b65a9415670e114f610/8a4e8/kafka5.png 150w,\n/static/38cc5152b7b77b65a9415670e114f610/5a46d/kafka5.png 300w,\n/static/38cc5152b7b77b65a9415670e114f610/f88d0/kafka5.png 493w\"\n        sizes=\"(max-width: 493px) 100vw, 493px\"\n        style=\"width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;\"\n        loading=\"lazy\"\n      />\n  </a>\n    </span></p>\n<h2 id=\"topics\" style=\"position:relative;\"><a href=\"#topics\" aria-label=\"topics permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Topics</h2>\n<p><span\n      class=\"gatsby-resp-image-wrapper\"\n      style=\"position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 600px; \"\n    >\n      <a\n    class=\"gatsby-resp-image-link\"\n    href=\"/static/4df39aa432c37afc1743174bb6d0b5ed/cdef6/topic.png\"\n    style=\"display: block\"\n    target=\"_blank\"\n    rel=\"noopener\"\n  >\n    <span\n    class=\"gatsby-resp-image-background-image\"\n    style=\"padding-bottom: 50%; position: relative; bottom: 0; left: 0; background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAKCAYAAAC0VX7mAAAACXBIWXMAAA7DAAAOwwHHb6hkAAABv0lEQVQoz21SW0/bMBjtH+YZTcC/4HEV2gP7AQxRaKVNQoDoSFW20qQkXXNpsjZxrk0vaW4H220pRT3Wkf3Z/o6P7a/iui7CMITvB3AcBywOgoDSxyyeYDoJEfou5nwcIQo8zKcxOMoSn1EpigI55QYlbSyeLhLIJoGk21saNkTaq2MfaZ7zffE8AfEjkCCiOUtUmEhGF5ShjfarBm3kceEn2cLB6SWOqzUcnTVwclbHUfUaX77WcFhtgEQzvu+vSnD10EXtUUJXd1eCyyxHU9ToZBfPioXQc/DzdxfSwOSe8zzDJAqRpsvVYT0DPbpG7BFu2wq+NwSc1wUIr9baYV5gYDno9E3o1OHINHBz38a/4Xj9VCV/5yzLePynb+FF0eC7Dn61ZHy7aVEKaErDrcP7zgA/7jp4knQksxiiNsYVvUZbHkKgjp77/9GSTRqbuHgQEcSL1ZUVE/WmyNnTHVTY6fme32JgH8bcz+YL6IaBSTzFMs34/Mb5LsqVQyZYMK43JkkC27ZBCIFHy4iVk+d5u6lcrOT9ltgKbiYZ0jTl9chEmLCqqny8m1y+C3/kjuC+a1iWxZ2ynjn9KLIPb8ek8QkoInI7AAAAAElFTkSuQmCC'); background-size: cover; display: block;\"\n  ></span>\n  <img\n        class=\"gatsby-resp-image-image\"\n        alt=\"Topics\"\n        title=\"Topics\"\n        src=\"/static/4df39aa432c37afc1743174bb6d0b5ed/0a47e/topic.png\"\n        srcset=\"/static/4df39aa432c37afc1743174bb6d0b5ed/8a4e8/topic.png 150w,\n/static/4df39aa432c37afc1743174bb6d0b5ed/5a46d/topic.png 300w,\n/static/4df39aa432c37afc1743174bb6d0b5ed/0a47e/topic.png 600w,\n/static/4df39aa432c37afc1743174bb6d0b5ed/1cfc2/topic.png 900w,\n/static/4df39aa432c37afc1743174bb6d0b5ed/cdef6/topic.png 1163w\"\n        sizes=\"(max-width: 600px) 100vw, 600px\"\n        style=\"width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;\"\n        loading=\"lazy\"\n      />\n  </a>\n    </span></p>\n<h2 id=\"patitions\" style=\"position:relative;\"><a href=\"#patitions\" aria-label=\"patitions permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Patitions</h2>\n<ul>\n<li>A topic consists 1 or more <code class=\"language-text\">partitions</code></li>\n<li>Each partition is <code class=\"language-text\">ordered</code>, <code class=\"language-text\">immutable</code> sequence of messages that is continually appended to a commit log</li>\n</ul>\n<h2 id=\"kafka-nouns\" style=\"position:relative;\"><a href=\"#kafka-nouns\" aria-label=\"kafka nouns permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Kafka nouns</h2>\n<ul>\n<li>topic, producer, consumer, broker, cluster,</li>\n<li>zookeeper, storm library, pub/sub</li>\n<li>Kafka is comparable to (相当于) traditional messaging systems such as <code class=\"language-text\">ActiveMQ</code></li>\n<li>topics, patitions, Replicas, replication, in-sync replicas</li>\n<li>leader, follower</li>\n<li>load balance</li>\n<li>consumer group</li>\n<li>fault tolerance</li>\n<li>log compaction, log cleaner</li>\n<li>delivery semantics</li>\n</ul>\n<blockquote>\n<p>There are 3 ways topic can be created</p>\n</blockquote>\n<p><span\n      class=\"gatsby-resp-image-wrapper\"\n      style=\"position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 600px; \"\n    >\n      <a\n    class=\"gatsby-resp-image-link\"\n    href=\"/static/3c3685df7ab033696d76fce826c67faf/38124/topic_creation.png\"\n    style=\"display: block\"\n    target=\"_blank\"\n    rel=\"noopener\"\n  >\n    <span\n    class=\"gatsby-resp-image-background-image\"\n    style=\"padding-bottom: 30%; position: relative; bottom: 0; left: 0; background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAGCAYAAADDl76dAAAACXBIWXMAAA7DAAAOwwHHb6hkAAABD0lEQVQY022Q22qEMBRF/TNBsc5YfdDxJ1vw+j29UPpi6yVG420G3T2JloHS4GKd7OQcVI3xFkkSI0lTpEQcx4iiCFmWUZaoTNb7GeXpE5L4mXr2+3meqx7pZVmgFUUB13XhOA7CMITnebAsC8HlAj/w4fs7p/OZ7j0iCE6wbQOGYcI0TTzYNnRdV31CCGgTTRWix7Zt2NYVkD7qldgOZA567qw764FaG7TldsNHzfBSVngta7xXjfLbYclX16OfZ3TTjHacVP13z6cJo/xkOfCzYWD0ukwMO8OA9uA3b8iSuhcq5+NIHim718NMA+frFaxl4G2LruPoOAdrGrWXbuoKZfmtXJFlLvoe/y76LT9DKJ1/96kgRgAAAABJRU5ErkJggg=='); background-size: cover; display: block;\"\n  ></span>\n  <img\n        class=\"gatsby-resp-image-image\"\n        alt=\"Topic Creation\"\n        title=\"Topic Creation\"\n        src=\"/static/3c3685df7ab033696d76fce826c67faf/0a47e/topic_creation.png\"\n        srcset=\"/static/3c3685df7ab033696d76fce826c67faf/8a4e8/topic_creation.png 150w,\n/static/3c3685df7ab033696d76fce826c67faf/5a46d/topic_creation.png 300w,\n/static/3c3685df7ab033696d76fce826c67faf/0a47e/topic_creation.png 600w,\n/static/3c3685df7ab033696d76fce826c67faf/1cfc2/topic_creation.png 900w,\n/static/3c3685df7ab033696d76fce826c67faf/38124/topic_creation.png 953w\"\n        sizes=\"(max-width: 600px) 100vw, 600px\"\n        style=\"width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;\"\n        loading=\"lazy\"\n      />\n  </a>\n    </span></p>\n<h3 id=\"data-serialization-formats\" style=\"position:relative;\"><a href=\"#data-serialization-formats\" aria-label=\"data serialization formats permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Data serialization formats</h3>\n<ul>\n<li>Kafka does not care about data format for a message payload</li>\n<li>it is up to developer to handle serialization/deserialization.</li>\n<li>common choices in practice: Avro, JSON</li>\n</ul>\n<h2 id=\"kafka-connect\" style=\"position:relative;\"><a href=\"#kafka-connect\" aria-label=\"kafka connect permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Kafka connect</h2>\n<p><span\n      class=\"gatsby-resp-image-wrapper\"\n      style=\"position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 600px; \"\n    >\n      <a\n    class=\"gatsby-resp-image-link\"\n    href=\"/static/4e4e255ce196ee3fe7efd2e04b5f47da/5205c/kafka-connect.png\"\n    style=\"display: block\"\n    target=\"_blank\"\n    rel=\"noopener\"\n  >\n    <span\n    class=\"gatsby-resp-image-background-image\"\n    style=\"padding-bottom: 62.66666666666667%; position: relative; bottom: 0; left: 0; background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAANCAYAAACpUE5eAAAACXBIWXMAAA7DAAAOwwHHb6hkAAACX0lEQVQ4y4WTS2/TQBSF/WfZseEPsGCBkBASC8QOseiuFVUFEqigCopoSts0SpOGRnk0ie04jh+x4ziPxvZ8jO2msdhwpSOP7Tt37j3njKKqKpquE8xmpCGEyPC/2OTESYLtutn+9Jti2jaW47BerwuJYlv4H4gi7vM9x2Y29bP9yrP9r7z6dIRpWdsO5alsNtyD4rowxTqK+N3VuOgMWCyXKIfXHU7aKp7nMfV9krRYYSNJLA/Ika4pHJCGZox48n6Px+92qfV1lORuwd08bzeOY1zXIez8gX4TMewhTA3GOmIDtQ16l+XEyXhzJh5vvvzgxcE3WsYYpV7e5ezXHtPplDAMCSQXq/IxXEoYPfzBLYN6FXwnL351iqiesDQG204LHCt6r8I8nGajpvAnE8LyT6iVYNTHkwWNZoO5IYtZQ7ipIBpl5iONxWpFX9PYLVU4OK9iez6KEywksTk3kXy6UrF5/QJaVRatOoEcXyuX8Jt1VvI9qZ1C45yZOcwsow4NXn484vXn7xiOi/L0wyE7x2cEQSDH9jMeNzG2LWKp4mTispbdWNaYKM5Fy6aRQm7stgnl0dsdnu8fMpvPuVcGYRvgmvidG6KRit9tEkn+pr02iWPKf2OZF2XpqYiObeKYbbyJiXLZVWnqowdfibuVVPE2F0BylkjyzesqYa8F8qBM6fR/GOQ+XEfUK1+olHZQ+w2Uwl3KVVstEbKQ8BzwbPm0qZ2d0ruuwUz6NP2eTrAIH1T2PStDNnKcqbt1vkgLSp8Jo591mkjfWVfnrG9vQK4zH6ZIJync6c1d+gs8k9afEP3LfwAAAABJRU5ErkJggg=='); background-size: cover; display: block;\"\n  ></span>\n  <img\n        class=\"gatsby-resp-image-image\"\n        alt=\"Kafka connect\"\n        title=\"Kafka connect\"\n        src=\"/static/4e4e255ce196ee3fe7efd2e04b5f47da/0a47e/kafka-connect.png\"\n        srcset=\"/static/4e4e255ce196ee3fe7efd2e04b5f47da/8a4e8/kafka-connect.png 150w,\n/static/4e4e255ce196ee3fe7efd2e04b5f47da/5a46d/kafka-connect.png 300w,\n/static/4e4e255ce196ee3fe7efd2e04b5f47da/0a47e/kafka-connect.png 600w,\n/static/4e4e255ce196ee3fe7efd2e04b5f47da/5205c/kafka-connect.png 833w\"\n        sizes=\"(max-width: 600px) 100vw, 600px\"\n        style=\"width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;\"\n        loading=\"lazy\"\n      />\n  </a>\n    </span></p>\n<ul>\n<li><code class=\"language-text\">Connectors</code> – the high level abstraction that coordinates data streaming by managing tasks</li>\n<li><code class=\"language-text\">Tasks</code> – the implementation of how data is copied to or from Kafka</li>\n<li><code class=\"language-text\">Workers</code> – the running processes that execute connectors and tasks: standlone and distributed.</li>\n<li><code class=\"language-text\">Converters</code> – the code used to translate data between Connect and the system sending or receiving data</li>\n<li><code class=\"language-text\">Transforms</code> – simple logic to alter each message produced by or sent to a connector</li>\n</ul>\n<h3 id=\"standard-confluent-connectors\" style=\"position:relative;\"><a href=\"#standard-confluent-connectors\" aria-label=\"standard confluent connectors permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Standard confluent connectors</h3>\n<p><span\n      class=\"gatsby-resp-image-wrapper\"\n      style=\"position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 600px; \"\n    >\n      <a\n    class=\"gatsby-resp-image-link\"\n    href=\"/static/a1fd5fbb95bac792ece823c8bcb3f176/394f7/kafka-connectors.png\"\n    style=\"display: block\"\n    target=\"_blank\"\n    rel=\"noopener\"\n  >\n    <span\n    class=\"gatsby-resp-image-background-image\"\n    style=\"padding-bottom: 60%; position: relative; bottom: 0; left: 0; background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAMCAYAAABiDJ37AAAACXBIWXMAAA7DAAAOwwHHb6hkAAAB+0lEQVQoz01S227TQBT0fwNFQiqUT+BfqJAq8dCmgYoHHpoEaBPb8foSO76vL8Oc4ziKpZF312fnzMyx8+X3Cz793ODz0x9cL1e4Wqxw87TBDc8+/ljjmnhz/4wPy7XWvH9cK65O7xnvHp7xlnB+hSm+bba4++tiGWa49xPc/XPx4Me6fzQpFibD922gdbfrV3xdvej79rQXyL1FkMIZO4ssNiizA/TpOxQp1+OAy6erK7RVgb6puC4xtI2eNfkRtipZMdU7eV1j53vwggBl06Dg3sQx7DCi6TqFHQYceDErS/0udXlVKew4Ktq+Z20Pp+LHMIoQJwlqrpu2RXY8oidJxyKBrAuSmTBEGE+1JgoRcW3ZcK4TOBU7CuHALgIhTWi557o7kcr5Mc+VJDBGUdKuRnFqOMMpKPt1tyVJyoMRotgX+2xkWdyKAhbK953nYW8CjWjrudi6LjI2ku9SZ5m/Wk4Oh3OHlkOSvVgfLhSK5TiJ1W5rLXpVNqvrz3BEScBsZulVUyPixTkTUSn2JVdR5wd7zU2a2IvszhmKZcMM58x0SAxbVFwSpiR0aXXPOKRhTsUiwJ5qZkyWOYTGtpNlEsWHRC3PkxZCIYjiyXIQGo1lmvD8NwwKp+ZFb++za6IEQiQKZdqzAhlWXhTqRJqFJBbrMpDL30vwHyskjvWAgPXhAAAAAElFTkSuQmCC'); background-size: cover; display: block;\"\n  ></span>\n  <img\n        class=\"gatsby-resp-image-image\"\n        alt=\"Standard confluent connectors\"\n        title=\"Standard confluent connectors\"\n        src=\"/static/a1fd5fbb95bac792ece823c8bcb3f176/0a47e/kafka-connectors.png\"\n        srcset=\"/static/a1fd5fbb95bac792ece823c8bcb3f176/8a4e8/kafka-connectors.png 150w,\n/static/a1fd5fbb95bac792ece823c8bcb3f176/5a46d/kafka-connectors.png 300w,\n/static/a1fd5fbb95bac792ece823c8bcb3f176/0a47e/kafka-connectors.png 600w,\n/static/a1fd5fbb95bac792ece823c8bcb3f176/394f7/kafka-connectors.png 707w\"\n        sizes=\"(max-width: 600px) 100vw, 600px\"\n        style=\"width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;\"\n        loading=\"lazy\"\n      />\n  </a>\n    </span></p>\n<h2 id=\"system-tools\" style=\"position:relative;\"><a href=\"#system-tools\" aria-label=\"system tools permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>System Tools</h2>\n<ul>\n<li>Kafka Manager</li>\n<li>Consumer Offset Checker</li>\n<li>Dump Log Segment</li>\n<li>Export Zookeeper Offsets</li>\n<li>Get Offset Shell</li>\n<li>Import Zookeeper Offsets</li>\n<li>JMX Tool</li>\n<li>Kafka Migration Tool</li>\n<li>Mirror Maker</li>\n<li>Replay Log Producer</li>\n<li>Simple Consumer Shell</li>\n<li>State Change Log Merger</li>\n<li>Update Offsets In Zookeeper</li>\n<li>Verify Consumer Rebalance</li>\n</ul>\n<h2 id=\"monitoring--configuration\" style=\"position:relative;\"><a href=\"#monitoring--configuration\" aria-label=\"monitoring  configuration permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Monitoring &#x26; Configuration</h2>\n<p>Use of standard monitoring tools is recommended</p>\n<ul>\n<li>\n<p>Graphite</p>\n<ul>\n<li>Puppet module: <a href=\"https://github.com/miguno/puppet-graphite\">https://github.com/miguno/puppet-graphite</a></li>\n<li>Java API, also used by Kafka: <a href=\"http://metrics.codahale.com/\">http://metrics.codahale.com/</a> </li>\n</ul>\n</li>\n<li>\n<p>JMX</p>\n<ul>\n<li><a href=\"https://kafka.apache.org/documentation.html#monitoring\">https://kafka.apache.org/documentation.html#monitoring</a> </li>\n</ul>\n</li>\n</ul>\n<p>Collect logging files into a central place</p>\n<ul>\n<li>Logstash/Kibana and friends</li>\n<li>Helps with troubleshooting, debugging, etc. – notably if you can correlate logging data with numeric metrics</li>\n</ul>\n<h2 id=\"qa\" style=\"position:relative;\"><a href=\"#qa\" aria-label=\"qa permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Q/A</h2>\n<ul>\n<li>ISR: Intra-cluster Replication</li>\n</ul>\n<h2 id=\"ecosystem\" style=\"position:relative;\"><a href=\"#ecosystem\" aria-label=\"ecosystem permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Ecosystem</h2>\n<ul>\n<li><a href=\"https://cwiki.apache.org/confluence/display/KAFKA/Ecosystem\">https://cwiki.apache.org/confluence/display/KAFKA/Ecosystem</a></li>\n<li>\n<p>zookeeper</p>\n<div class=\"gatsby-highlight\" data-language=\"bash\"><pre style=\"counter-reset: linenumber NaN\" class=\"language-bash line-numbers\"><code class=\"language-bash\">$ kafka-topics.sh --zookeeper zookeeper1:2181 --create --topic zerg.hydra <span class=\"token punctuation\">\\</span>\u000b    --partitions <span class=\"token number\">3</span> --replication-factor <span class=\"token number\">2</span> --config <span class=\"token assign-left variable\">x</span><span class=\"token operator\">=</span>y</code><span aria-hidden=\"true\" class=\"line-numbers-rows\" style=\"white-space: normal; width: auto; left: 0;\"><span></span></span></pre></div>\n</li>\n<li><a href=\"https://istio.io/\">istio</a></li>\n</ul>","tableOfContents":"<ul>\n<li>\n<p><a href=\"#kafka\">Kafka</a></p>\n<ul>\n<li><a href=\"#what-is-kafka\">What is Kafka</a></li>\n<li>\n<p><a href=\"#what-does-kafka-do\">What does Kafka do?</a></p>\n<ul>\n<li><a href=\"#qualities\">Qualities</a></li>\n<li><a href=\"#high-level-overview\">High-level overview</a></li>\n<li><a href=\"#kafka-architecture\">Kafka architecture</a></li>\n<li><a href=\"#core-apis\">Core APIs</a></li>\n</ul>\n</li>\n<li><a href=\"#topics\">Topics</a></li>\n<li><a href=\"#patitions\">Patitions</a></li>\n<li>\n<p><a href=\"#kafka-nouns\">Kafka nouns</a></p>\n<ul>\n<li><a href=\"#data-serialization-formats\">Data serialization formats</a></li>\n</ul>\n</li>\n<li>\n<p><a href=\"#kafka-connect\">Kafka connect</a></p>\n<ul>\n<li><a href=\"#standard-confluent-connectors\">Standard confluent connectors</a></li>\n</ul>\n</li>\n<li><a href=\"#system-tools\">System Tools</a></li>\n<li><a href=\"#monitoring--configuration\">Monitoring &#x26; Configuration</a></li>\n<li><a href=\"#qa\">Q/A</a></li>\n<li><a href=\"#ecosystem\">Ecosystem</a></li>\n</ul>\n</li>\n</ul>"},"previous":{"fields":{"id":"36884170-cd04-548a-9a00-77e36a4b1380","title":"2 Hadoop","slug":"/bigdata/2-hadoop/"},"excerpt":"Hadoop The Apache Hadoop software library is a framework that allows for the distributed processing of large data sets across clusters of…"},"next":{"fields":{"id":"f2387a56-a470-54c8-94aa-182fffbab8b0","title":"201 Nosql 8","slug":"/bigdata/201-nosql-8/"},"excerpt":"1. RDBMS vs NoSQL 2. CAP THEOREM (上限定理) \n 3. NOSQL DB TYPES SQL/New SQL Document DB: semi-structured data Column Based DB: ETL, data…"}},"pageContext":{"id":"c9cdff49-84d1-58e6-bf25-926b7f68d84c","prevId":"36884170-cd04-548a-9a00-77e36a4b1380","nextId":"f2387a56-a470-54c8-94aa-182fffbab8b0"}},"staticQueryHashes":["1576573137","63159454"]}